"use client" import * as React from 'react' import { Play, ShieldCheck } from 'lucide-react' import { Drawer, DrawerContent, DrawerHeader, DrawerTitle, DrawerDescription, DrawerBody, DrawerFooter, DrawerClose, } from '@open-mercato/ui/primitives/drawer' import { Button } from '@open-mercato/ui/primitives/button' import { Input } from '@open-mercato/ui/primitives/input' import { Label } from '@open-mercato/ui/primitives/label' import { Spinner } from '@open-mercato/ui/primitives/spinner' import { SwitchField } from '@open-mercato/ui/primitives/switch-field' import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@open-mercato/ui/primitives/select' import { apiCall, apiCallOrThrow } from '@open-mercato/ui/backend/utils/apiCall' import { flash } from '@open-mercato/ui/backend/FlashMessages' import { useGuardedMutation } from '@open-mercato/ui/backend/injection/useGuardedMutation' import { useT } from '@open-mercato/shared/lib/i18n/context' import { shortCaseId } from '../../../../components/subjectRef' type ApprovedCase = { id: string processType: string | null } export type RunEvaluationDrawerProps = { open: boolean onOpenChange: (open: boolean) => void agentId: string agentLabel: string onStarted: () => void } const ALL_CASES = '__all__' const FORM_ID = 'agent-orchestrator-run-evaluation-form' function readString(record: Record, ...keys: string[]): string { for (const key of keys) { const value = record[key] if (typeof value === 'string') return value } return '' } /** * Triggers an evaluation run (`POST /eval-runs`) for the host agent against its * approved eval cases. Eval runs are `source=eval`: they replay fresh inference * under the agent's own principal, propose-only, and never touch production * metrics or caseload. */ export function RunEvaluationDrawer({ open, onOpenChange, agentId, agentLabel, onStarted }: RunEvaluationDrawerProps) { const t = useT() const [cases, setCases] = React.useState([]) const [isLoading, setIsLoading] = React.useState(false) const [selection, setSelection] = React.useState(ALL_CASES) const [repeatCount, setRepeatCount] = React.useState(1) const [judgeMayGate, setJudgeMayGate] = React.useState(false) const [isBusy, setIsBusy] = React.useState(false) const { runMutation, retryLastMutation } = useGuardedMutation<{ retryLastMutation: () => Promise }>({ contextId: 'agent_orchestrator.agentDetail.runEvaluation', blockedMessage: t('agent_orchestrator.evalCases.flash.actionError'), }) React.useEffect(() => { if (!open) return setSelection(ALL_CASES) setRepeatCount(1) setJudgeMayGate(false) let cancelled = false setIsLoading(true) void apiCall<{ items?: Array> }>( `/api/agent_orchestrator/eval-cases?agentDefinitionId=${encodeURIComponent(agentId)}&status=approved&pageSize=100`, undefined, { fallback: { items: [] } }, ).then((call) => { if (cancelled) return const items = Array.isArray(call.result?.items) ? call.result.items : [] setCases( items .map((item): ApprovedCase | null => { const id = readString(item, 'id') if (!id) return null return { id, processType: readString(item, 'process_type', 'processType') || null } }) .filter((row): row is ApprovedCase => !!row), ) setIsLoading(false) }) return () => { cancelled = true } }, [open, agentId]) const submit = React.useCallback(async () => { const evalCaseIds = selection === ALL_CASES ? cases.map((row) => row.id) : [selection] if (evalCaseIds.length === 0) { flash(t('agent_orchestrator.agentDetail.evaluation.run.noApproved', 'This agent has no approved cases to evaluate.'), 'error') return } setIsBusy(true) try { await runMutation({ operation: () => apiCallOrThrow<{ suiteRunId?: string }>( '/api/agent_orchestrator/eval-runs', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ agentDefinitionId: agentId, evalCaseIds, repeatCount, judgeMayGate, }), }, ), context: { retryLastMutation }, }) flash(t('agent_orchestrator.agentDetail.evaluation.run.started', 'Evaluation started.'), 'success') onOpenChange(false) onStarted() } catch (err) { flash(err instanceof Error ? err.message : t('agent_orchestrator.evalCases.flash.actionError'), 'error') } finally { setIsBusy(false) } }, [selection, cases, agentId, repeatCount, judgeMayGate, runMutation, retryLastMutation, onOpenChange, onStarted, t]) const noApproved = !isLoading && cases.length === 0 return ( }> {t('agent_orchestrator.agentDetail.evaluation.section.runs', 'Runs')} {t('agent_orchestrator.agentDetail.evaluation.run.title', 'Run evaluation')} {agentLabel || agentId}
{t( 'agent_orchestrator.agentDetail.evaluation.run.sourceNote', 'Eval runs are source=eval — fresh, propose-only inference that never touches production metrics or caseload.', )}
{isLoading ? (
{t('agent_orchestrator.agentDetail.evaluation.run.loadingCases', 'Loading approved cases…')}
) : noApproved ? (

{t('agent_orchestrator.agentDetail.evaluation.run.noApproved', 'This agent has no approved cases to evaluate.')}

) : (
{ event.preventDefault(); void submit() }} onKeyDown={(event) => { if ((event.metaKey || event.ctrlKey) && event.key === 'Enter') { event.preventDefault() void submit() } }} >
{ const next = Number(event.target.value) setRepeatCount(Number.isFinite(next) ? Math.min(20, Math.max(1, Math.round(next))) : 1) }} />

{t('agent_orchestrator.agentDetail.evaluation.run.repeatHint', 'Replay each case this many times to measure model/judge variance.')}

)}
) } export default RunEvaluationDrawer